<?php
/**
 * Client.php
 * 描述
 *
 * 作者:  胡毅 (huyi@comsenz.com)
 * 创建时间: 2012-10-13 下午2:17:39
 * 修改记录:
 *
 * $id$
 */

require_once(APP_PATH . '/vendors/dom/simple_html_dom.php');


class Flight_Crawl_Client {

    /**
     * _config
     * 爬虫配置
     *
     * @var array
     */
    private $_config = array();

    /**
     * _city
     * 城市列表
     *
     * @var array
     * + cityId
     * + cityName
     */
    private $_city = array();

    /**
     * _false
     * 重试次数
     *
     * @var integer
     */
    private $_retry = 0;

    /**
     * _charset
     * 字符集
     *
     * @var string
     */
    private $_charset = 'UTF-8';

    /**
     * _handle
     * log文件句柄
     *
     */
    private $_handle;

    /**
     * _logPath
     *
     * @var string
     */
    private $_logPath = '/tmp/log_bankcoupon_';

    /**
     * __construct
     *
     * @param  array $conifg
     * @return void
     */
    public function __construct($conifg = array()) {

        if (!$conifg) {
            throw new Exception('config can not be empty');
        }
        $this->_handle = fopen($this->_logPath . date('Y-m-d') . '.txt', 'a+');
        $this->_config = $conifg;
        $this->_view = new simple_html_dom();

    }

    /**
     * __destruct
     *
     * @return void
     */
    public function __destruct() {
        fclose($this->_handle);
    }

    /**
     * run
     *
     * @return void
     */
    public function run() {

        $html = $this->load($this->_config['city']['url'], $this->_config['city']['method'], $this->_config['city']['params']);
        $this->_city = $this->getAllCity($html, $this->_config['city']);
        // 增加一次重试，广发行偶尔第一次会返回一个空页面
        if (!$this->_city) {
            $html = $this->load($this->_config['city']['url'], $this->_config['city']['method'], $this->_config['city']['params']);
            $this->_city = $this->getAllCity($html, $this->_config['city']);
        }
        $arguments = array();

        // 如果配置了重试次数,则初始值为0，否则为1
        if ($this->_config['conf']['retry']) {
            $initRetry = 0;
        } else {
            $initRetry = 1;
        }
        // 配置laod参数
        $listConfig = array(
                            'url' => $this->_config['list']['url'],
                            'method' => $this->_config['list']['method'],
                            'params' => $this->_config['list']['postData'],
                           );

        foreach ($this->_city as $city) {

            try {
                // 重置重试次数
                $this->_retry = $initRetry;
                // 替换url中的参数
                $url = $this->_config['list']['url'];
                foreach ($city as $k => $v) {
                    $url = str_replace('{$' . $k . '}', $v, $url);
                }
                $listConfig['url'] = $url;

                // 城市信息当作arguments处理
                $arguments = array_merge($arguments, $city);
                $listConfig['params'] = array_merge($listConfig['params'], $city, $arguments);
                $listResult = $this->crawlListByCity($listConfig, $arguments);

            } catch (Exception $e) {
                $this->addFailLog($city[$this->_config['city']['fieldMap']['name']], '抓取列表遇到异常，退出' . $e->getMessage());
            }
        }

        $this->addSuccessLog('', '===end===');
    }

    /**
     * load
     *
     * @param  mixed $url
     * @param  string $method
     * @param  array $params
     * @return void
     */
    public function load($url, $method = 'get', $params = array()) {

        try {
            set_time_limit(0);
            $proxy = 'http://10.130.24.42:80/';
            $ch = curl_init();
            curl_setopt($ch, CURLOPT_HTTPPROXYTUNNEL, true);
            curl_setopt($ch, CURLOPT_PROXY, $proxy);
            curl_setopt($ch, CURLOPT_URL, $url);
            curl_setopt($ch, CURLOPT_PORT, 80);
            curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
            // 设置超时
            curl_setopt($ch, CURLOPT_TIMEOUT, 6);

            if (strtolower($method) == 'post') {
                $params = @http_build_query($params);
                curl_setopt($ch, CURLOPT_POST, 1);
                curl_setopt($ch, CURLOPT_POSTFIELDS, $params);
            } else {
                if (is_array($params) && array_values($params)) {
                    $url = $url . "?" . @http_build_query($params);
                    curl_setopt($ch, CURLOPT_URL, $url);
                }
            }

            $result = curl_exec($ch);
            $curTime = 0;
            // 单个页面失败重试10次
            while($curTime < 10) {
                if ($result) {
                    break;
                }
                $result = curl_exec($ch);
                $curTime++;
            }
            $result = mb_convert_encoding($result, $this->_charset, $this->_config['conf']['charset']);

            curl_close($ch);
            //        $this->_view->load($str);
        } catch (Exception $e) {
            return false;
        }

        return $result;
    }

    /**
     * getAllCity
     *
     * @param  mixed $url
     * @param  mixed $pattern
     * @return void
     */
    public function getAllCity($html, $cityConfig) {

        $this->_city = $this->_parseHtml($html, $cityConfig['crawlInfo']);
        $key = key($this->_city);
        $cityList = array();

        for($i=0 ; $i < count($this->_city[$key]); $i++) {

            // 转为各个网站适用的城市参数
            foreach ($this->_config['city']['fieldMap'] as $k => $v) {
                if(array_key_exists($k, $this->_city) && array_key_exists($i, $this->_city[$k])) {
                    $cityList[$i][$v] = $this->_city[$k][$i];
                } else {
                    $cityList[$i][$v] = '';
                }
            }
        }

        return $cityList;
    }

    /**
     * crawlListByCity
     *
     * @param  mixed $loadConfig
     * @param  array $arguments
     * @return void
     */
    public function crawlListByCity($loadConfig, $arguments = array()) {

        $cityName = $arguments[$this->_config['city']['fieldMap']['name']];

        $result = $this->load($loadConfig['url'], $loadConfig['method'], $loadConfig['params']);

        $result = $this->_parseList($result, $this->_config['list'], $arguments);

        // 生成新config
        if ($loadConfig['method'] == 'post') {
            $loadConfig['params'] = $result['page'];
        } else {
            $loadConfig['params'] = $result['page'];
//            $loadConfig['url'] = $result['page'];
        }

        // 如果抓取失败，重试
        if (!$result && $this->_retry >= $this->_config['conf']['retry']) {
            $this->addFailLog($cityName, "\t抓取列表失败, 已经重试" . $this->_retry . "次");
            return false;
        } elseif (($result['page'] || !$result['result']) && $this->_retry < $this->_config['conf']['retry']) {
            $this->_retry++;
            $this->crawlListByCity($loadConfig, $arguments);
        }

        // start
        foreach ($result['result'] as $value) {

            try {
                usleep(rand(200000, 1000000));
                // 详情页暂不考虑post的情况
                if ($this->_config['detail']['url']) {
                    $detailHtml = $this->load($value, $this->_config['detail']['method']);
                    $arguments['detailUrl'] = $value;
                } else {
                    $detailHtml = $value;
                    $arguments['detailUrl'] = '';
                }

                // 得到插入数据库的数组
                $detailData = $this->getData($detailHtml, $arguments);

                // 数据入库
                $saveResult = $this->save($detailData);

                // 记录log
                if ($saveResult) {
                    $this->addSuccessLog($cityName, $arguments['detailUrl']);
                } else {
                    $this->addFailLog($cityName, $arguments['detailUrl']);
                }
            } catch (Exception $e){
                $this->addFailLog($cityName, $arguments['detailUrl'] . "\t" . $e->getMessage());
            }
        }
        // end

        // 如果page=false ，则到达尾页
        if ($result['page'] === false && $this->_retry >= $this->_config['conf']['retry']) {
            return false;
        }

        // 递归抓取列表
        $this->crawlListByCity($loadConfig, $arguments);
    }

    /**
     * find
     * 找到对应的HTML标签
     *
     * @param unknown_type $selector 标签名字
     * @param unknown_type $idx 是第几个
     * @param unknown_type $lowercase
     */
    function find($selector, $idx=null, $lowercase=false) {

        return $this->_view->find($selector, $idx, $lowercase);
    }

    /**
     * _parseList
     * 解析列表页html，返回值随配置的改变而有所不同
     * 1.详情页页抓取方式为get时，result为包含详情页url的数组
     *  ep:'result' => array(
     *                  0 => 'http://www.xxxx.com',
     *                  .....
     *                  )
     * 1.详情页页抓取方式为post时，result为包含详情页用到的参数数组
     *  ep:'result' => array(
     *                      0 => array(
     *                              'id' => '999999',
     *                              .....
     *                          ),
     *                      .....
     *                      );
     * 3.如果没有详情页,则返回多个html(类似于一个详情页)
     *  ep: 'result' => array(
     *                      0 => '<div>...',
     *                      .....
     *                  )
     * 4.如果列表抓取方式为get,page为下一页的url链接
     * 5.如果列表抓取方式为post,page为包含键=>值的参数数组
     * 6.最后一页没有下一页的链接,返回false
     *
     * @param  string $contents
     * @param  array $listConfig
     * @param  array $arguments
     * @return array
     *  + result array|string
     *  + page array|bolean
     */
    private function _parseList($contents, $listConfig, $arguments = array()) {

        // 需要根据返回值判断有无下一页
        $crawlConfig = $listConfig['crawlInfo'];
        $crawlInfos = $this->_parseHtml($contents, $crawlConfig);
        foreach ($arguments as $key => $value) {
            if(!array_key_exists($key, $crawlInfos)) {
                $crawlInfos[$key] = $value;
            }
        }
        if($listConfig['method'] === 'post') {
            $pageConfig = $listConfig['postData'];
            $crawlInfos['page'] = array();
            foreach ($pageConfig as $pageOne => $value) {
                if(array_key_exists($pageOne, $crawlInfos) && $crawlInfos[$pageOne]) {
                    $crawlInfos['page'][$pageOne] = $crawlInfos[$pageOne];
                    unset($crawlInfos[$pageOne]);
                    continue;
                }
                $crawlInfos['page'] = false;
                break;
            }
        } else {
            $pageConfig = $listConfig['postData'];
            $crawlInfos['page'] = array();
            $pageKey = $listConfig['page'][0];
            foreach ($pageConfig as $pageOne => $value) {
                $crawlInfos['page'][$pageOne] = $crawlInfos[$pageOne];
                unset($crawlInfos[$pageOne]);
            }
            foreach ($listConfig['postData'] as $k => $v) {
                if ($v) {
                    $crawlInfos['page'][$k] = $v;
                }
            }
            if (!$crawlInfos['page'][$pageKey]) {
                $crawlInfos['page'] = false;
            }
        }
        //$urlIdMethod = $this->_config['detail']['method'];
        //if($urlIdMethod === 'get') {
        $urlIdConfig = $this->_config['detail']['url'];
        $crawlInfos['result'] = $crawlInfos['id'];
        unset($crawlInfos['id']);
        if($urlIdConfig) {
            foreach ($crawlInfos['result'] as $key => $value) {
                $crawlInfos['result'][$key] = $urlIdConfig . $value;
            }
        }
        return $crawlInfos;
        //}

        //$postData = $this->_config['detail']['postData'];
        //foreach ($postData as $value) {
        //    $crawlInfos['result'][$value] = $crawlInfos[$value];
        //    unset($crawlInfos[$value]);
        //}
        //return $crawlInfos;
    }

    /**
     * _parseHtml
     * 分析网页
     *
     * @param unknown_type $contents
     * @param unknown_type $crawlInfo
     */
    private function _parseHtml($contents, $crawlInfo) {

        // 需要返回的信息
        $returnInfo = array();
        $crawlContents = $contents;
        foreach ($crawlInfo as $name => $params) {
            foreach ($params as $value) {
                $method = key($value);
                $param = $value[$method];
                try {
                    $crawlContents = $this->_parseAdapter($crawlContents, $param, $method);
                } catch (Exception $e) {
                    Logger::error($e);
                }
            }
            if($name === 'pre') {
                $contents = $crawlContents;
                continue;
            }
            $returnInfo[$name] = $crawlContents;
            $crawlContents = $contents;
        }
        return $returnInfo;
    }

    /**
     * _parseAdapter
     * 解析适配器
     *
     * @param unknown_type $contents
     * @param unknown_type $param
     * @param unknown_type $name
     */
    private function _parseAdapter($contents, $param, $name) {

        $functionName = '_parse' . $name;
        if(!method_exists($this, $functionName)) {
            throw new Meishi_Crawl_Exception('config is error:' . $functionName . ' can not parse');
        }
        if(is_array($contents)) {
            foreach ($contents as $key => $value) {
                $contents[$key] = $this->$functionName($value, $param);
            }
            return $contents;
        }
        $contents = $this->$functionName($contents, $param);
        return $contents;
    }

    /**
     * _parseCallback
     * 处理回调函数
     *
     * @param  mixed $contents
     * @param  mixed $method
     * @return void
     */
    private function _parseCallback($contents, $method) {

        return call_user_func($method[0], $contents);
    }

    /**
     * _parseReplace
     * 解析Replace
     *
     * @param string $contents
     * @param array $param
     * @return string
     */
    private function _parseReplace($contents, $param) {

        return trim(preg_replace($param[0], $param[1], $contents));
    }

    /**
     * _parseMatch
     * 解析match
     *
     * @param string $contents
     * @param array $param
     * @return string|array
     */
    private function _parseMatch($contents, $param) {

        if($param[0]) {
            preg_match_all($param[1], $contents, $contents);
            if($param[2] === 'all') {
                if($contents[0]) {
                    foreach ($contents[0] as $key => $value) {
                        $contents[0][$key] = trim($value);
                    }
                    return $contents[0];
                }
                return array();
            }
            if(array_key_exists($param[2], $contents[0])) {
                if($contents[0][$param[2]]) {
                    return trim($contents[0][$param[2]]);
                }
            }
            return '';
        } else {
            if(preg_match($param[1], $contents, $contents)) {
                if($param[2] === 'all') {
                    return trim($contents[0]);
                }
                if(array_key_exists($param[2], $contents)) {
                    return trim($contents[$param[2]]);
                }
            }
            return '';
        }
    }

    /**
     * _parseFind
     * 解析find，支持无穷参数，但仅有倒数第二个参数可以是all或者all-a
     *
     * @param array $param
     * @return string
     */
    private function _parseFind($contents, $param) {

        $this->_view->load($contents);
        $paramNumber = count($param);
        if($param[$paramNumber - 2] === 'all-a') {
            $crawlContents = array();
            $param[$paramNumber - 2] = 'all';
        } else {
            $crawlContents = '';
        }

        if($param[1] !== 'all') {

            // 进行第一次find
            $findObject = $this->find($param[0], $param[1]);

            // 进行n次find，直至只剩3个参数
            for ($i = 2; $i < $paramNumber - 3; $i += 2) {
                $findObject = $findObject->find($param[$i], $param[$i+1]);
            }

            // 倒数第二个参数为All
            if($param[$paramNumber - 2] === 'all') {
                foreach ($findObject->find($param[$paramNumber - 3]) as $value) {
                    if(is_array($crawlContents)) {
                        $crawlContents[] = trim($value->$param[$paramNumber - 1]);
                        continue;
                    }
                    $crawlContents .= trim($value->$param[$paramNumber - 1]);
                }
            } else {
                if($paramNumber > 3) {
                    $findObject = $findObject->find($param[$paramNumber - 3], $param[$paramNumber - 2]);
                }
                $crawlContents = trim($findObject->$param[$paramNumber - 1]);
            }
        } else {

            // 仅有3个参数，且第二个参数为ALL
            foreach ($this->find($param[0]) as $value) {
                if(is_array($crawlContents)) {
                    $crawlContents[] = trim($value->$param[$paramNumber - 1]);
                    continue;
                }
                $crawlContents .= trim($value->$param[2]);
            }
        }
        return $crawlContents;
    }

    /**
     * getData
     * 取得详细页数据
     *
     * @param  mixed $html
     * @param  array $arguments
     * @return array
     */
    public function getData($html, $arguments = array()) {

        $arguments = array_merge($arguments, $this->_config['detail']['arguments']);
        $data = $this->_parseData($html, $this->_config['detail'], $arguments);

        return $data;
    }

    /**
     * _parseData
     *
     * @param  string $contents html信息
     * @param  array $viewConfig 详情页配置
     * @param  array $arguments 需要传进来的值
     * @return  array
     */
    private function _parseData($contents, $viewConfig, $arguments = array()) {

        $crawlConfig = $viewConfig['crawlInfo'];
        $crawlInfos = $this->_parseHtml($contents, $crawlConfig);
        if(array_key_exists('detailUrl', $arguments)) {
            $crawlInfos['bcoShopUrl'] = $arguments['detailUrl'];
        }
        if(array_key_exists('detailUrl', $arguments) && !array_key_exists('bcoShopId', $crawlInfos)) {
            $result = $this->_parseHtml($arguments['detailUrl'], $viewConfig['shopId']);
            $crawlInfos['bcoShopId'] = $result['id'];
        }
        if(!array_key_exists('bcoCity', $crawlInfos)) {
            $crawlInfos['bcoCity'] = $arguments[$this->_config['city']['fieldMap']['name']];
        }
        if(!array_key_exists('bcoBankName', $crawlInfos)) {
            $crawlInfos['bcoBankName'] = $this->_config['conf']['bankName'];
        }
        if(!array_key_exists('bcoHtml', $crawlInfos)) {
            $crawlInfos['bcoHtml'] = $contents;
        }
        if(array_key_exists('bcoStartTime', $crawlInfos)) {
            $crawlInfos['bcoStartTime'] = date('Y-m-d', strtotime($crawlInfos['bcoStartTime']));
        }
        if(array_key_exists('bcoEndTime', $crawlInfos)) {
            $crawlInfos['bcoEndTime'] = date('Y-m-d', strtotime($crawlInfos['bcoEndTime']));
        }
        return $crawlInfos;
    }

    /**
     * save
     * 数据入库
     *
     * @param  array $data
     * @return void
     */
    public function save($data = array()) {

        // save data to db
        try {
            // 如果关键字段没有取到则跳过
            $requiredFields = array('bcoShopName', 'bcoShopId', 'bcoCouponInfo', 'bcoCardInfo', 'bcoAddress', 'bcoTel');
            foreach ($requiredFields as $field) {
                if (!$data[$field]) {
                    return false;
                }
            }
            $data['bcoHash'] = md5($data['bcoShopId'] . $data['bcoCouponInfo'] . $data['bcoCardInfo']);
            $bankCouponService =& Service::factory('Meishi_Service_BankCoupon');
            $result = $bankCouponService->saveCoupon($data);
        }catch (Exception $e) {
            return false;
        }

        return true;
    }

    /**
     * addFailLog
     *
     * @param  string $msg
     * @return void
     */
    protected function addFailLog($city, $msg = '') {

        return fwrite($this->_handle, date("Y-m-d H:i:s") . "\tfail\t"  . $this->_config['conf']['bankName'] . "\t{$city}\t" . $msg . "\n");
    }

    /**
     * addSuccessLog
     *
     * @param  string $msg
     * @return void
     */
    protected function addSuccessLog($city, $msg = '') {

        return fwrite($this->_handle, date("Y-m-d H:i:s") . "\tsuccess\t" . $this->_config['conf']['bankName'] . "\t{$city}\t" . $msg . "\n");
    }
}
